{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import GPUtil\n",
    "from threading import Thread\n",
    "import time\n",
    "\n",
    "attrList = [[{'attr':'id','name':'ID'}],\n",
    "            [{'attr':'temperature','name':'GPU temp.','suffix':'C','transform': lambda x: x,'precision':0},\n",
    "             {'attr':'load','name':'GPU util.','suffix':'%','transform': lambda x: x*100,'precision':0},\n",
    "             {'attr':'memoryUtil','name':'Memory util.','suffix':'%','transform': lambda x: x*100,'precision':0}],\n",
    "            [{'attr':'memoryTotal','name':'Memory total','suffix':'MB','precision':0},\n",
    "             {'attr':'memoryUsed','name':'Memory used','suffix':'MB','precision':0},\n",
    "             {'attr':'memoryFree','name':'Memory free','suffix':'MB','precision':0}]]\n",
    "\n",
    "class Monitor(Thread):\n",
    "    def __init__(self, delay):\n",
    "        super(Monitor, self).__init__()\n",
    "        self.stopped = False\n",
    "        self.delay = delay # Time between calls to GPUtil\n",
    "        self.start()\n",
    "\n",
    "    def run(self):\n",
    "        while not self.stopped:\n",
    "#             GPUtil.showUtilization(all=True)\n",
    "            GPUtil.showUtilization(attrList=attrList)\n",
    "            time.sleep(self.delay)\n",
    "\n",
    "    def stop(self):\n",
    "        self.stopped = True\n",
    "        \n",
    "# Instantiate monitor with a 10-second delay between updates\n",
    "monitor = Monitor(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       99% |          78% ||      16130MB |     12557MB |      3573MB |\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       70C |       99% |          49% ||      16130MB |      7931MB |      8199MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          77% ||      16130MB |     12467MB |      3663MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          47% ||      16130MB |      7657MB |      8473MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       99% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          40% ||      16130MB |      6523MB |      9607MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          34% ||      16130MB |      5551MB |     10579MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          26% ||      16130MB |      4201MB |     11929MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          48% ||      16130MB |      7751MB |      8379MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |       95% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       99% |          67% ||      16130MB |     10811MB |      5319MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          76% ||      16130MB |     12287MB |      3843MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       97% |          54% ||      16130MB |      8701MB |      7429MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       98% |          35% ||      16130MB |      5641MB |     10489MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12555MB |      3575MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          27% ||      16130MB |      4417MB |     11713MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12555MB |      3575MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          38% ||      16130MB |      6181MB |      9949MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       99% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       98% |          33% ||      16130MB |      5353MB |     10777MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       95% |          26% ||      16130MB |      4273MB |     11857MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          48% ||      16130MB |      7823MB |      8307MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       95% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       99% |          71% ||      16130MB |     11531MB |      4599MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          60% ||      16130MB |      9657MB |      6473MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          53% ||      16130MB |      8485MB |      7645MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |       99% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       97% |          39% ||      16130MB |      6253MB |      9877MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       74C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       71C |       97% |          26% ||      16130MB |      4183MB |     11947MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          58% ||      16130MB |      9317MB |      6813MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |      100% |          78% ||      16130MB |     12557MB |      3573MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       73C |       99% |          72% ||      16130MB |     11585MB |      4545MB |\n",
      "| ID || GPU temp. | GPU util. | Memory util. || Memory total | Memory used | Memory free |\n",
      "------------------------------------------------------------------------------------------\n",
      "|  0 ||       72C |       95% |          65% ||      16130MB |     10409MB |      5721MB |\n"
     ]
    }
   ],
   "source": [
    "# monitor.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_pytorch_transformers)",
   "language": "python",
   "name": "conda_pytorch_transformers"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
